{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f9feb00b",
   "metadata": {},
   "source": [
    "# LSTM + LSTM\n",
    "\n",
    "采用 LSTM 进行Encoder 操作， Dense 进行预测\n",
    "\n",
    "可尝试采用单层、多层的LSTM，结果可采用output、hidden进行尝试\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1bc57651",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "from seq2seq_v105 import Seq2Seq\n",
    " \n",
    "from data_process.sort_process import SortProcess\n",
    "from data_process.data_iter import DataIter \n",
    "\n",
    "\n",
    "src_path = \"data/sort/letters_source.txt\"\n",
    "trg_path = \"data/sort/letters_target.txt\"\n",
    "\n",
    "sp = SortProcess(src_path, trg_path)\n",
    "src_ids, trg_ids, label_ids = sp.dataset(max_len=10, to_numpy=False)\n",
    "src_ids = np.array(src_ids)\n",
    "trg_ids = np.array(trg_ids)\n",
    "label_ids = np.array(label_ids) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8ee7447e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-08-08 23:40:43.049163: W tensorflow/core/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n",
      "2022-08-08 23:40:47.554723: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:40:47.555056: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:40:47.555395: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:40:47.555581: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:40:47.555983: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:40:47.556158: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:40:47.556528: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:40:47.556706: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:40:47.557062: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:40:47.557272: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "313/313 [==============================] - 19s 40ms/step - loss: 1.1607 - acc: 0.6514\n",
      "Epoch 2/10\n",
      "313/313 [==============================] - 13s 41ms/step - loss: 0.3952 - acc: 0.8751\n",
      "Epoch 3/10\n",
      "313/313 [==============================] - 13s 41ms/step - loss: 0.1140 - acc: 0.9699\n",
      "Epoch 4/10\n",
      "313/313 [==============================] - 13s 42ms/step - loss: 0.0460 - acc: 0.9908\n",
      "Epoch 5/10\n",
      "313/313 [==============================] - 14s 46ms/step - loss: 0.0270 - acc: 0.9957\n",
      "Epoch 6/10\n",
      "313/313 [==============================] - 14s 45ms/step - loss: 0.0137 - acc: 0.9984\n",
      "Epoch 7/10\n",
      "313/313 [==============================] - 13s 42ms/step - loss: 0.0236 - acc: 0.9950\n",
      "Epoch 8/10\n",
      "313/313 [==============================] - 13s 42ms/step - loss: 0.0050 - acc: 0.9998\n",
      "Epoch 9/10\n",
      "313/313 [==============================] - 14s 43ms/step - loss: 0.0048 - acc: 0.9995\n",
      "Epoch 10/10\n",
      "313/313 [==============================] - 13s 41ms/step - loss: 0.0258 - acc: 0.9927\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x14b13b400>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
    "loss = tf.keras.losses.SparseCategoricalCrossentropy()\n",
    "model = Seq2Seq(vocab_size=len(sp.i2c))\n",
    "model.compile(optimizer=opt, loss=loss, metrics=['acc'])\n",
    "model.fit(x=[src_ids, trg_ids],\n",
    "          y=label_ids, \n",
    "          batch_size=32,\n",
    "          epochs=10,\n",
    "          callbacks=[], \n",
    "         )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a74ee4c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27ce4fdc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "77be8828",
   "metadata": {},
   "outputs": [],
   "source": [
    "from data_process.sort_process import SortProcess\n",
    "from data_process.data_iter import DataIter \n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "964b55fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "src_path = \"data/sort/letters_source.txt\"\n",
    "trg_path = \"data/sort/letters_target.txt\"\n",
    "\n",
    "sp = SortProcess(src_path, trg_path)\n",
    "src_ids, trg_ids, label_ids = sp.dataset(max_len=10, to_numpy=False)\n",
    "src_ids = np.array(src_ids)\n",
    "trg_ids = np.array(trg_ids)\n",
    "label_ids = np.array(label_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "147bbc12",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "id": "45b430f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_dim=128 \n",
    "lstm_units=256\n",
    "vocab_size = 30\n",
    "\n",
    "encoder_embed = tf.keras.layers.Embedding(input_dim=vocab_size, output_dim=embed_dim)\n",
    "decoder_embed = tf.keras.layers.Embedding(input_dim=vocab_size, output_dim=embed_dim)\n",
    "\n",
    "encoder_lstm = tf.keras.layers.LSTM(units=lstm_units, return_state=True,\n",
    "                                    return_sequences=True)  \n",
    "\n",
    "decoder_lstm = tf.keras.layers.LSTM(units=lstm_units, return_state=True,\n",
    "                                    return_sequences=False)\n",
    "attention = tf.keras.layers.Attention(use_scale=True)\n",
    "\n",
    "dense = tf.keras.layers.Dense(units=vocab_size, activation='softmax')\n",
    "\n",
    "\n",
    "i_src = np.random.randint(0, 30, (32, 10))\n",
    "i_trg = np.random.randint(0, 30, (32, 10))\n",
    "\n",
    "i_e = encoder_embed(i_n)   \n",
    "outputs, hidden, cell = encoder_lstm(i_e)\n",
    "\n",
    "\n",
    "for t in range(i_src.shape[1]): \n",
    "    x = tf.slice(i_src, [0, t], [-1, 1])  # 每次取一个steps    \n",
    "    d_e = decoder_embed(x) \n",
    "    out_put, hidden, cell = decoder_lstm(d_e, initial_state=[hidden, cell]) \n",
    "    \n",
    "    out_put = tf.expand_dims(out_put, -2)\n",
    "    att_text = attention(inputs=[out_put, outputs])\n",
    "    \n",
    "    out = dense(out_put)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb81d369",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4bb958f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "d74584ca",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "\n",
    "class Seq2Seq(tf.keras.models.Model):\n",
    "    \"\"\" 简单的Seq2Seq模型\n",
    "    Encoder: Embedding + LSTM\n",
    "    Decoder: Embedding + LSTM + Dense\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, vocab_size, embed_dim=128, lstm_units=256):\n",
    "        super().__init__()\n",
    "\n",
    "        # 定义 Encoder、Decoder EMbedding\n",
    "        self.encoder_embed = tf.keras.layers.Embedding(input_dim=vocab_size,\n",
    "                                                       output_dim=embed_dim)\n",
    "        self.decoder_embed = tf.keras.layers.Embedding(input_dim=vocab_size,\n",
    "                                                       output_dim=embed_dim)\n",
    "\n",
    "        # 定义Encoder、Decoder LSTM\n",
    "        self.encoder_lstm = tf.keras.layers.LSTM(units=lstm_units, return_state=True)\n",
    "        self.bidirect_layer = tf.keras.layers.Bidirectional(self.encoder_lstm, merge_mode='sum')\n",
    "\n",
    "        self.decoder_lstm = tf.keras.layers.LSTM(units=lstm_units, return_state=True)\n",
    "        self.attention = tf.keras.layers.Attention(use_scale=True)\n",
    "\n",
    "        # 定义全连接层\n",
    "        self.dense = tf.keras.layers.Dense(units=vocab_size, activation='softmax')\n",
    "\n",
    "    def call(self, inputs, start_index=2, training=True):\n",
    "        \"\"\" 计算，运算\n",
    "        inputs: 输入数据，\n",
    "            train: [src, trg]\n",
    "            predict: src\n",
    "        \"\"\"\n",
    "        if training:\n",
    "            src_b, trg_b = inputs\n",
    "        else:\n",
    "            src_b = inputs\n",
    "        e_e = self.encoder_embed(src_b)\n",
    "        outputs, f_hid, f_cell, b_hid, b_cell = self.bidirect_layer(e_e)\n",
    "        hidden = tf.add(f_hid, b_hid)\n",
    "        cell = tf.add(f_cell, b_cell)\n",
    "        \n",
    "        t_outputs = []\n",
    "        for t in range(src_b.shape[1]): \n",
    "            if training or t == 0:\n",
    "                # t== 0 这是一个隐患，当 src 开始 和 trget input 开始 index 不一致的时候，会有问题\n",
    "                x = tf.slice(src_b, [0, t], [-1, 1])  # 每次取一个steps\n",
    "            d_e = self.decoder_embed(x)\n",
    "            out_put, hidden, cell = self.decoder_lstm(d_e, initial_state=[hidden, cell]) \n",
    "            out_put = tf.expand_dims(out_put, -2) \n",
    "            att_text = self.attention(inputs=[out_put, outputs]) \n",
    "            out_put = tf.add(out_put, att_text) \n",
    "            out = self.dense(out_put)\n",
    "            if not training:\n",
    "                x = tf.argmax(out, axis=-1) \n",
    "            t_outputs.append(out)\n",
    "            \n",
    "        f_out = tf.concat(t_outputs, axis=1)\n",
    "        return f_out\n",
    "\n",
    "opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
    "loss = tf.keras.losses.SparseCategoricalCrossentropy()\n",
    "model = Seq2Seq(vocab_size=len(sp.i2c))\n",
    "model.compile(optimizer=opt, loss=loss, metrics=['acc'])\n",
    "# model.fit(x=[src_ids, trg_ids], y=label_ids, batch_size=32, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c286d1f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-08-08 23:34:46.034812: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:34:46.035017: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:34:46.035245: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:34:46.035433: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:34:46.035618: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:34:46.035824: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:34:46.035996: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:34:46.036184: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:34:46.036364: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-08-08 23:34:46.036540: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 2s 2s/step\n"
     ]
    }
   ],
   "source": [
    "t = \"dfsab\"\n",
    "t_c = [sp.beg_flag] + list(t) + [sp.end_flag]\n",
    "t_index = sp.to_index(t_c)\n",
    "i_p = np.array([t_index], dtype=np.int64)\n",
    "i_p = tf.convert_to_tensor(i_p, dtype=tf.int64)\n",
    "\n",
    "p = model.predict(i_p)\n",
    "# p = model([i_p, i_p])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85c66fb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "p_id = tf.argmax(p, axis=-1).numpy()\n",
    "\n",
    "[sp.i2c.get(i, '?') for i in p_id[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d9b6f39",
   "metadata": {},
   "outputs": [],
   "source": [
    "a = tf.convert_to_tensor([[1, 1]])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "56607ea8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-31 18:03:59.728753: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-07-31 18:03:59.728963: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-07-31 18:03:59.729318: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-07-31 18:03:59.729646: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-07-31 18:03:59.729830: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-07-31 18:03:59.730250: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-07-31 18:03:59.730429: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-07-31 18:03:59.730799: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-07-31 18:03:59.730990: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n",
      "2022-07-31 18:03:59.731287: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: \"Softmax\" attr { key: \"T\" value { type: DT_FLOAT } } inputs { dtype: DT_FLOAT shape { unknown_rank: true } } device { type: \"CPU\" model: \"0\" num_cores: 8 environment { key: \"cpu_instruction_set\" value: \"ARM NEON\" } environment { key: \"eigen\" value: \"3.4.90\" } l1_cache_size: 16384 l2_cache_size: 524288 l3_cache_size: 524288 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { unknown_rank: true } }\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "313/313 [==============================] - 17s 37ms/step - loss: 1.1399 - acc: 0.6635\n",
      "Epoch 2/4\n",
      "313/313 [==============================] - 11s 35ms/step - loss: 0.2953 - acc: 0.9104\n",
      "Epoch 3/4\n",
      "313/313 [==============================] - 11s 35ms/step - loss: 0.0956 - acc: 0.9761\n",
      "Epoch 4/4\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.0396 - acc: 0.9936\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "in user code:\n\n    File \"/Users/st/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/engine/training.py\", line 1845, in predict_function  *\n        return step_function(self, iterator)\n    File \"/Users/st/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/engine/training.py\", line 1834, in step_function  **\n        outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"/Users/st/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/engine/training.py\", line 1823, in run_step  **\n        outputs = model.predict_step(data)\n    File \"/Users/st/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/engine/training.py\", line 1791, in predict_step\n        return self(x, training=False)\n    File \"/Users/st/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/utils/traceback_utils.py\", line 67, in error_handler\n        raise e.with_traceback(filtered_tb) from None\n    File \"/var/folders/mk/sf1l6n495zv6mrxqq6t_v73h0000gn/T/__autograph_generated_fileoujlf0ca.py\", line 89, in tf__call\n        ag__.for_stmt(ag__.converted_call(ag__.ld(range), (ag__.ld(src_b).shape[1],), None, fscope), None, loop_body, get_state_3, set_state_3, ('cell', 'hidden', 'x'), {'iterate_names': 't'})\n    File \"/var/folders/mk/sf1l6n495zv6mrxqq6t_v73h0000gn/T/__autograph_generated_fileoujlf0ca.py\", line 60, in loop_body\n        (out_put, hidden, cell) = ag__.converted_call(ag__.ld(self).decoder_lstm, (ag__.ld(d_e),), dict(initial_state=[ag__.ld(hidden), ag__.ld(cell)]), fscope)\n\n    ValueError: Exception encountered when calling layer \"seq2_seq_4\" (type Seq2Seq).\n    \n    in user code:\n    \n        File \"/Users/st/Desktop/UP/nnm/keras/seq2seq/seq2seq_v104.py\", line 48, in call  *\n            out_put, hidden, cell = self.decoder_lstm(d_e, initial_state=[hidden, cell])\n        File \"/Users/st/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/layers/rnn/base_rnn.py\", line 573, in __call__  **\n            return super(RNN, self).__call__(inputs, **kwargs)\n        File \"/Users/st/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/utils/traceback_utils.py\", line 67, in error_handler\n            raise e.with_traceback(filtered_tb) from None\n        File \"/Users/st/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/engine/input_spec.py\", line 214, in assert_input_compatibility\n            raise ValueError(f'Input {input_index} of layer \"{layer_name}\" '\n    \n        ValueError: Input 0 of layer \"lstm_12\" is incompatible with the layer: expected ndim=3, found ndim=4. Full shape received: (None, 1, 1, 128)\n    \n    \n    Call arguments received by layer \"seq2_seq_4\" (type Seq2Seq):\n      • inputs=tf.Tensor(shape=(None, 10), dtype=int64)\n      • start_index=2\n      • training=False\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Input \u001b[0;32mIn [11]\u001b[0m, in \u001b[0;36m<cell line: 15>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m i_p \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39marray([t_index], dtype\u001b[38;5;241m=\u001b[39mnp\u001b[38;5;241m.\u001b[39mint64)\n\u001b[1;32m     13\u001b[0m i_p \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mconvert_to_tensor(i_p, dtype\u001b[38;5;241m=\u001b[39mtf\u001b[38;5;241m.\u001b[39mint64)\n\u001b[0;32m---> 15\u001b[0m p \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpredict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mi_p\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     17\u001b[0m p_id \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39margmax(p, axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mnumpy()\n\u001b[1;32m     19\u001b[0m [sp\u001b[38;5;241m.\u001b[39mi2c\u001b[38;5;241m.\u001b[39mget(i, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m?\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m p_id[\u001b[38;5;241m0\u001b[39m]]\n",
      "File \u001b[0;32m~/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/utils/traceback_utils.py:67\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     65\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:  \u001b[38;5;66;03m# pylint: disable=broad-except\u001b[39;00m\n\u001b[1;32m     66\u001b[0m   filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[0;32m---> 67\u001b[0m   \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28mNone\u001b[39m\n\u001b[1;32m     68\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m     69\u001b[0m   \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n",
      "File \u001b[0;32m/var/folders/mk/sf1l6n495zv6mrxqq6t_v73h0000gn/T/__autograph_generated_file_tl3bf1i.py:15\u001b[0m, in \u001b[0;36mouter_factory.<locals>.inner_factory.<locals>.tf__predict_function\u001b[0;34m(iterator)\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     14\u001b[0m     do_return \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m---> 15\u001b[0m     retval_ \u001b[38;5;241m=\u001b[39m ag__\u001b[38;5;241m.\u001b[39mconverted_call(ag__\u001b[38;5;241m.\u001b[39mld(step_function), (ag__\u001b[38;5;241m.\u001b[39mld(\u001b[38;5;28mself\u001b[39m), ag__\u001b[38;5;241m.\u001b[39mld(iterator)), \u001b[38;5;28;01mNone\u001b[39;00m, fscope)\n\u001b[1;32m     16\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[1;32m     17\u001b[0m     do_return \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n",
      "File \u001b[0;32m/var/folders/mk/sf1l6n495zv6mrxqq6t_v73h0000gn/T/__autograph_generated_fileoujlf0ca.py:89\u001b[0m, in \u001b[0;36mouter_factory.<locals>.inner_factory.<locals>.tf__call\u001b[0;34m(self, inputs, start_index, training)\u001b[0m\n\u001b[1;32m     87\u001b[0m out \u001b[38;5;241m=\u001b[39m ag__\u001b[38;5;241m.\u001b[39mUndefined(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mout\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     88\u001b[0m att_text \u001b[38;5;241m=\u001b[39m ag__\u001b[38;5;241m.\u001b[39mUndefined(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124matt_text\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m---> 89\u001b[0m ag__\u001b[38;5;241m.\u001b[39mfor_stmt(ag__\u001b[38;5;241m.\u001b[39mconverted_call(ag__\u001b[38;5;241m.\u001b[39mld(\u001b[38;5;28mrange\u001b[39m), (ag__\u001b[38;5;241m.\u001b[39mld(src_b)\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m1\u001b[39m],), \u001b[38;5;28;01mNone\u001b[39;00m, fscope), \u001b[38;5;28;01mNone\u001b[39;00m, loop_body, get_state_3, set_state_3, (\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcell\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhidden\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mx\u001b[39m\u001b[38;5;124m'\u001b[39m), {\u001b[38;5;124m'\u001b[39m\u001b[38;5;124miterate_names\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt\u001b[39m\u001b[38;5;124m'\u001b[39m})\n\u001b[1;32m     90\u001b[0m f_out \u001b[38;5;241m=\u001b[39m ag__\u001b[38;5;241m.\u001b[39mconverted_call(ag__\u001b[38;5;241m.\u001b[39mld(tf)\u001b[38;5;241m.\u001b[39mconcat, (ag__\u001b[38;5;241m.\u001b[39mld(t_outputs),), \u001b[38;5;28mdict\u001b[39m(axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m), fscope)\n\u001b[1;32m     91\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n",
      "File \u001b[0;32m/var/folders/mk/sf1l6n495zv6mrxqq6t_v73h0000gn/T/__autograph_generated_fileoujlf0ca.py:60\u001b[0m, in \u001b[0;36mouter_factory.<locals>.inner_factory.<locals>.tf__call.<locals>.loop_body\u001b[0;34m(itr)\u001b[0m\n\u001b[1;32m     58\u001b[0m ag__\u001b[38;5;241m.\u001b[39mif_stmt(ag__\u001b[38;5;241m.\u001b[39mor_((\u001b[38;5;28;01mlambda\u001b[39;00m : ag__\u001b[38;5;241m.\u001b[39mld(training)), (\u001b[38;5;28;01mlambda\u001b[39;00m : (ag__\u001b[38;5;241m.\u001b[39mld(t) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m))), if_body_1, else_body_1, get_state_1, set_state_1, (\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mx\u001b[39m\u001b[38;5;124m'\u001b[39m,), \u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m     59\u001b[0m d_e \u001b[38;5;241m=\u001b[39m ag__\u001b[38;5;241m.\u001b[39mconverted_call(ag__\u001b[38;5;241m.\u001b[39mld(\u001b[38;5;28mself\u001b[39m)\u001b[38;5;241m.\u001b[39mdecoder_embed, (ag__\u001b[38;5;241m.\u001b[39mld(x),), \u001b[38;5;28;01mNone\u001b[39;00m, fscope)\n\u001b[0;32m---> 60\u001b[0m (out_put, hidden, cell) \u001b[38;5;241m=\u001b[39m \u001b[43mag__\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconverted_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43mag__\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mld\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecoder_lstm\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43mag__\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mld\u001b[49m\u001b[43m(\u001b[49m\u001b[43md_e\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mdict\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43minitial_state\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43mag__\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mld\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhidden\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mag__\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mld\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcell\u001b[49m\u001b[43m)\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfscope\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     61\u001b[0m out_put \u001b[38;5;241m=\u001b[39m ag__\u001b[38;5;241m.\u001b[39mconverted_call(ag__\u001b[38;5;241m.\u001b[39mld(tf)\u001b[38;5;241m.\u001b[39mexpand_dims, (ag__\u001b[38;5;241m.\u001b[39mld(out_put), (\u001b[38;5;241m-\u001b[39m \u001b[38;5;241m2\u001b[39m)), \u001b[38;5;28;01mNone\u001b[39;00m, fscope)\n\u001b[1;32m     62\u001b[0m att_text \u001b[38;5;241m=\u001b[39m ag__\u001b[38;5;241m.\u001b[39mconverted_call(ag__\u001b[38;5;241m.\u001b[39mld(\u001b[38;5;28mself\u001b[39m)\u001b[38;5;241m.\u001b[39mattention, (), \u001b[38;5;28mdict\u001b[39m(inputs\u001b[38;5;241m=\u001b[39m[ag__\u001b[38;5;241m.\u001b[39mld(out_put), ag__\u001b[38;5;241m.\u001b[39mld(outputs)]), fscope)\n",
      "\u001b[0;31mValueError\u001b[0m: in user code:\n\n    File \"/Users/st/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/engine/training.py\", line 1845, in predict_function  *\n        return step_function(self, iterator)\n    File \"/Users/st/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/engine/training.py\", line 1834, in step_function  **\n        outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"/Users/st/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/engine/training.py\", line 1823, in run_step  **\n        outputs = model.predict_step(data)\n    File \"/Users/st/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/engine/training.py\", line 1791, in predict_step\n        return self(x, training=False)\n    File \"/Users/st/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/utils/traceback_utils.py\", line 67, in error_handler\n        raise e.with_traceback(filtered_tb) from None\n    File \"/var/folders/mk/sf1l6n495zv6mrxqq6t_v73h0000gn/T/__autograph_generated_fileoujlf0ca.py\", line 89, in tf__call\n        ag__.for_stmt(ag__.converted_call(ag__.ld(range), (ag__.ld(src_b).shape[1],), None, fscope), None, loop_body, get_state_3, set_state_3, ('cell', 'hidden', 'x'), {'iterate_names': 't'})\n    File \"/var/folders/mk/sf1l6n495zv6mrxqq6t_v73h0000gn/T/__autograph_generated_fileoujlf0ca.py\", line 60, in loop_body\n        (out_put, hidden, cell) = ag__.converted_call(ag__.ld(self).decoder_lstm, (ag__.ld(d_e),), dict(initial_state=[ag__.ld(hidden), ag__.ld(cell)]), fscope)\n\n    ValueError: Exception encountered when calling layer \"seq2_seq_4\" (type Seq2Seq).\n    \n    in user code:\n    \n        File \"/Users/st/Desktop/UP/nnm/keras/seq2seq/seq2seq_v104.py\", line 48, in call  *\n            out_put, hidden, cell = self.decoder_lstm(d_e, initial_state=[hidden, cell])\n        File \"/Users/st/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/layers/rnn/base_rnn.py\", line 573, in __call__  **\n            return super(RNN, self).__call__(inputs, **kwargs)\n        File \"/Users/st/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/utils/traceback_utils.py\", line 67, in error_handler\n            raise e.with_traceback(filtered_tb) from None\n        File \"/Users/st/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/engine/input_spec.py\", line 214, in assert_input_compatibility\n            raise ValueError(f'Input {input_index} of layer \"{layer_name}\" '\n    \n        ValueError: Input 0 of layer \"lstm_12\" is incompatible with the layer: expected ndim=3, found ndim=4. Full shape received: (None, 1, 1, 128)\n    \n    \n    Call arguments received by layer \"seq2_seq_4\" (type Seq2Seq):\n      • inputs=tf.Tensor(shape=(None, 10), dtype=int64)\n      • start_index=2\n      • training=False\n"
     ]
    }
   ],
   "source": [
    "import seq2seq_v104 as seq2seq\n",
    "\n",
    "opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
    "loss = tf.keras.losses.SparseCategoricalCrossentropy()\n",
    "model = seq2seq.Seq2Seq(vocab_size=len(sp.i2c))\n",
    "model.compile(optimizer=opt, loss=loss, metrics=['acc'])\n",
    "model.fit(x=[src_ids, trg_ids], y=label_ids, batch_size=32, epochs=4)\n",
    "\n",
    "t = \"dfsab\"\n",
    "t_c = [sp.beg_flag] + list(t) + [sp.end_flag]\n",
    "t_index = sp.to_index(t_c)\n",
    "i_p = np.array([t_index], dtype=np.int64)\n",
    "i_p = tf.convert_to_tensor(i_p, dtype=tf.int64)\n",
    "\n",
    "p = model.predict(i_p)\n",
    "\n",
    "p_id = tf.argmax(p, axis=-1).numpy()\n",
    "\n",
    "[sp.i2c.get(i, '?') for i in p_id[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc22eabd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5211442b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "b6f5751b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# x = tf.random.normal(shape=(32, 10))\n",
    "\n",
    "x = np.random.randint(0, 30, size=(32, 10))\n",
    "x = tf.convert_to_tensor(x, dtype=tf.int64)\n",
    "\n",
    "# model.predict(x)\n",
    "\n",
    "# x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "c5a6ae3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 10)"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i_p.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3292155",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "b843505a",
   "metadata": {},
   "outputs": [],
   "source": [
    "t = \"dfasf\"\n",
    "t_c = [sp.beg_flag] + list(t) + [sp.end_flag]\n",
    "t_index = sp.to_index(t_c)\n",
    "i_p = np.array([t_index], dtype=np.int64)\n",
    "i_p = tf.convert_to_tensor(i_p, dtype=tf.int64)\n",
    "\n",
    "# model.predict(i_p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "7d3d5f6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.call(i_p, training=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "fdca3ca7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.call(x, training=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "42f0ebe4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([32])"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.random.normal(shape=(32, 30))\n",
    "tf.argmax(x, axis=-1).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "9e040541",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = 0\n",
    "\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "326fc02e",
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.argmax?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ff9436b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "dc0f6eec",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = tf.random.normal(shape=(32, 10, 10))\n",
    "# y = tf.random.normal(shape=(32, 10))\n",
    "\n",
    "# xy = [x, y]\n",
    "\n",
    "# tf.concat(xy, axis=0).shape\n",
    "\n",
    "# tf.slice()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c7b21dd5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(10, 10), dtype=int64, numpy=\n",
       "array([[22, 20,  3,  2, 15, 15,  2, 26, 25, 27],\n",
       "       [ 3,  8,  6, 26,  4, 12, 12, 10,  8, 13],\n",
       "       [13, 29, 13, 15, 23, 26, 17, 26, 27, 30],\n",
       "       [22, 31,  4,  4, 16, 17, 21, 16,  0,  0],\n",
       "       [30,  9,  4, 14, 25, 29, 11,  2,  3, 28],\n",
       "       [30,  0,  8, 24, 13, 22, 24, 18, 22, 24],\n",
       "       [22, 15, 24,  1, 24, 12, 13,  7, 12, 10],\n",
       "       [19, 10, 18, 20,  7,  0, 26, 27, 19,  4],\n",
       "       [18, 14, 26, 16,  8, 21, 13, 22, 29, 31],\n",
       "       [17, 17,  9,  7, 10, 20, 23, 27, 22,  2]])>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.argmax(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fe912ace",
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.constant?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "45e93f80",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([32, 1])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.slice(x, [0, 0], [-1, 1]).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e8afe4ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tf.kaeras.optimizers.Adam?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3519e175",
   "metadata": {},
   "outputs": [],
   "source": [
    "  t = tf.constant([[[1, 1, 1], [2, 2, 2]],\n",
    "                   [[3, 3, 3], [4, 4, 4]],\n",
    "                   [[5, 5, 5], [6, 6, 6]]])\n",
    "  tf.slice(t, [1, 0, 0], [1, 1, 3])  # [[[3, 3, 3]]]\n",
    "  tf.slice(t, [1, 0, 0], [1, 2, 3])  # [[[3, 3, 3],\n",
    "                                     #   [4, 4, 4]]]\n",
    "  tf.slice(t, [1, 0, 0], [2, 1, 3])  # [[[3, 3, 3]],\n",
    "                                     #  [[5, 5, 5]]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a1cd25e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cbf6ba7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "22e49eab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "313/313 [==============================] - 14s 27ms/step - loss: 0.8964 - acc: 0.7002\n",
      "Epoch 2/10\n",
      "313/313 [==============================] - 9s 29ms/step - loss: 0.0906 - acc: 0.9713\n",
      "Epoch 3/10\n",
      "313/313 [==============================] - 9s 29ms/step - loss: 0.0128 - acc: 0.9972\n",
      "Epoch 4/10\n",
      "313/313 [==============================] - 9s 29ms/step - loss: 0.0030 - acc: 0.9995\n",
      "Epoch 5/10\n",
      "313/313 [==============================] - 9s 30ms/step - loss: 0.0071 - acc: 0.9983\n",
      "Epoch 6/10\n",
      "313/313 [==============================] - 9s 29ms/step - loss: 0.0070 - acc: 0.9981\n",
      "Epoch 7/10\n",
      " 11/313 [>.............................] - ETA: 8s - loss: 0.0119 - acc: 0.9955"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [7]\u001b[0m, in \u001b[0;36m<cell line: 32>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     30\u001b[0m model \u001b[38;5;241m=\u001b[39m Seq2Seq(vocab_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(sp\u001b[38;5;241m.\u001b[39mi2c))\n\u001b[1;32m     31\u001b[0m model\u001b[38;5;241m.\u001b[39mcompile(optimizer\u001b[38;5;241m=\u001b[39mopt, loss\u001b[38;5;241m=\u001b[39mloss, metrics\u001b[38;5;241m=\u001b[39m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124macc\u001b[39m\u001b[38;5;124m'\u001b[39m])\n\u001b[0;32m---> 32\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43msrc_ids\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrg_ids\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlabel_ids\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m32\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m10\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/utils/traceback_utils.py:64\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     63\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 64\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     65\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:  \u001b[38;5;66;03m# pylint: disable=broad-except\u001b[39;00m\n\u001b[1;32m     66\u001b[0m   filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
      "File \u001b[0;32m~/miniforge3/envs/tf2/lib/python3.8/site-packages/keras/engine/training.py:1409\u001b[0m, in \u001b[0;36mModel.fit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1402\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m tf\u001b[38;5;241m.\u001b[39mprofiler\u001b[38;5;241m.\u001b[39mexperimental\u001b[38;5;241m.\u001b[39mTrace(\n\u001b[1;32m   1403\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtrain\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m   1404\u001b[0m     epoch_num\u001b[38;5;241m=\u001b[39mepoch,\n\u001b[1;32m   1405\u001b[0m     step_num\u001b[38;5;241m=\u001b[39mstep,\n\u001b[1;32m   1406\u001b[0m     batch_size\u001b[38;5;241m=\u001b[39mbatch_size,\n\u001b[1;32m   1407\u001b[0m     _r\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m):\n\u001b[1;32m   1408\u001b[0m   callbacks\u001b[38;5;241m.\u001b[39mon_train_batch_begin(step)\n\u001b[0;32m-> 1409\u001b[0m   tmp_logs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_function\u001b[49m\u001b[43m(\u001b[49m\u001b[43miterator\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1410\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m data_handler\u001b[38;5;241m.\u001b[39mshould_sync:\n\u001b[1;32m   1411\u001b[0m     context\u001b[38;5;241m.\u001b[39masync_wait()\n",
      "File \u001b[0;32m~/miniforge3/envs/tf2/lib/python3.8/site-packages/tensorflow/python/util/traceback_utils.py:150\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    149\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 150\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    151\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    152\u001b[0m   filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
      "File \u001b[0;32m~/miniforge3/envs/tf2/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py:915\u001b[0m, in \u001b[0;36mFunction.__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    912\u001b[0m compiler \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mxla\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnonXla\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    914\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m OptionalXlaContext(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile):\n\u001b[0;32m--> 915\u001b[0m   result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    917\u001b[0m new_tracing_count \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexperimental_get_tracing_count()\n\u001b[1;32m    918\u001b[0m without_tracing \u001b[38;5;241m=\u001b[39m (tracing_count \u001b[38;5;241m==\u001b[39m new_tracing_count)\n",
      "File \u001b[0;32m~/miniforge3/envs/tf2/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py:947\u001b[0m, in \u001b[0;36mFunction._call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    944\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock\u001b[38;5;241m.\u001b[39mrelease()\n\u001b[1;32m    945\u001b[0m   \u001b[38;5;66;03m# In this case we have created variables on the first call, so we run the\u001b[39;00m\n\u001b[1;32m    946\u001b[0m   \u001b[38;5;66;03m# defunned version which is guaranteed to never create variables.\u001b[39;00m\n\u001b[0;32m--> 947\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_stateless_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# pylint: disable=not-callable\u001b[39;00m\n\u001b[1;32m    948\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_stateful_fn \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    949\u001b[0m   \u001b[38;5;66;03m# Release the lock early so that multiple threads can perform the call\u001b[39;00m\n\u001b[1;32m    950\u001b[0m   \u001b[38;5;66;03m# in parallel.\u001b[39;00m\n\u001b[1;32m    951\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock\u001b[38;5;241m.\u001b[39mrelease()\n",
      "File \u001b[0;32m~/miniforge3/envs/tf2/lib/python3.8/site-packages/tensorflow/python/eager/function.py:2453\u001b[0m, in \u001b[0;36mFunction.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2450\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock:\n\u001b[1;32m   2451\u001b[0m   (graph_function,\n\u001b[1;32m   2452\u001b[0m    filtered_flat_args) \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_maybe_define_function(args, kwargs)\n\u001b[0;32m-> 2453\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mgraph_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_flat\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   2454\u001b[0m \u001b[43m    \u001b[49m\u001b[43mfiltered_flat_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcaptured_inputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgraph_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcaptured_inputs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniforge3/envs/tf2/lib/python3.8/site-packages/tensorflow/python/eager/function.py:1860\u001b[0m, in \u001b[0;36mConcreteFunction._call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1856\u001b[0m possible_gradient_type \u001b[38;5;241m=\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPossibleTapeGradientTypes(args)\n\u001b[1;32m   1857\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (possible_gradient_type \u001b[38;5;241m==\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPOSSIBLE_GRADIENT_TYPES_NONE\n\u001b[1;32m   1858\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m executing_eagerly):\n\u001b[1;32m   1859\u001b[0m   \u001b[38;5;66;03m# No tape is watching; skip to running the function.\u001b[39;00m\n\u001b[0;32m-> 1860\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_build_call_outputs(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_inference_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1861\u001b[0m \u001b[43m      \u001b[49m\u001b[43mctx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcancellation_manager\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcancellation_manager\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m   1862\u001b[0m forward_backward \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_select_forward_and_backward_functions(\n\u001b[1;32m   1863\u001b[0m     args,\n\u001b[1;32m   1864\u001b[0m     possible_gradient_type,\n\u001b[1;32m   1865\u001b[0m     executing_eagerly)\n\u001b[1;32m   1866\u001b[0m forward_function, args_with_tangents \u001b[38;5;241m=\u001b[39m forward_backward\u001b[38;5;241m.\u001b[39mforward()\n",
      "File \u001b[0;32m~/miniforge3/envs/tf2/lib/python3.8/site-packages/tensorflow/python/eager/function.py:497\u001b[0m, in \u001b[0;36m_EagerDefinedFunction.call\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    495\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m _InterpolateFunctionError(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    496\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m cancellation_manager \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 497\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m \u001b[43mexecute\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    498\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msignature\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    499\u001b[0m \u001b[43m        \u001b[49m\u001b[43mnum_outputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_num_outputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    500\u001b[0m \u001b[43m        \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    501\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattrs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    502\u001b[0m \u001b[43m        \u001b[49m\u001b[43mctx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mctx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    503\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    504\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m execute\u001b[38;5;241m.\u001b[39mexecute_with_cancellation(\n\u001b[1;32m    505\u001b[0m         \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msignature\u001b[38;5;241m.\u001b[39mname),\n\u001b[1;32m    506\u001b[0m         num_outputs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_outputs,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    509\u001b[0m         ctx\u001b[38;5;241m=\u001b[39mctx,\n\u001b[1;32m    510\u001b[0m         cancellation_manager\u001b[38;5;241m=\u001b[39mcancellation_manager)\n",
      "File \u001b[0;32m~/miniforge3/envs/tf2/lib/python3.8/site-packages/tensorflow/python/eager/execute.py:54\u001b[0m, in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     52\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     53\u001b[0m   ctx\u001b[38;5;241m.\u001b[39mensure_initialized()\n\u001b[0;32m---> 54\u001b[0m   tensors \u001b[38;5;241m=\u001b[39m \u001b[43mpywrap_tfe\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTFE_Py_Execute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mctx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_handle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mop_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     55\u001b[0m \u001b[43m                                      \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_outputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     56\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m core\u001b[38;5;241m.\u001b[39m_NotOkStatusException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     57\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "class Seq2Seq(tf.keras.models.Model):\n",
    "    def __init__(self, vocab_size, embed_dim=128, lstm_units=256):\n",
    "        super().__init__()\n",
    "        self.encoder_embed = embedding = tf.keras.layers.Embedding(input_dim=vocab_size, \n",
    "                                                   output_dim=embed_dim)\n",
    "        self.decoder_embed = embedding = tf.keras.layers.Embedding(input_dim=vocab_size, \n",
    "                                                           output_dim=embed_dim)\n",
    "        self.encoder_lstm = tf.keras.layers.LSTM(units=lstm_units, return_state=True)\n",
    "        self.decoder_lstm = tf.keras.layers.LSTM(units=lstm_units, return_state=True)\n",
    "        self.dense = tf.keras.layers.Dense(units=vocab_size, activation='softmax')\n",
    "        \n",
    "    def call(self, inputs):\n",
    "        src_b, trg_b = inputs\n",
    "        e_e = self.encoder_embed(src_b)\n",
    "        _, hidden, cell = self.encoder_lstm(e_e)\n",
    "        t_outputs = []\n",
    "        for t in range(trg_ids.shape[1]):\n",
    "            x = trg_b[:, t]\n",
    "            x = tf.expand_dims(x, -1) # x.reshape(-1, 1)\n",
    "            d_e = self.decoder_embed(x)\n",
    "            out_put, hidden, cell = self.decoder_lstm(d_e, initial_state=[hidden, cell])\n",
    "            out = self.dense(out_put)\n",
    "            out = tf.expand_dims(out, -2)\n",
    "            t_outputs.append(out)\n",
    "        f_out = tf.concat(t_outputs, axis=1)\n",
    "        return f_out\n",
    "\n",
    "opt = tf.keras.optimizers.Adam(learning_rate=0.01)\n",
    "loss = tf.keras.losses.SparseCategoricalCrossentropy()\n",
    "model = Seq2Seq(vocab_size=len(sp.i2c))\n",
    "model.compile(optimizer=opt, loss=loss, metrics=['acc'])\n",
    "model.fit(x=[src_ids, trg_ids], y=label_ids, batch_size=32, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c74f3ba",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "422c6239",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "712ece75",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cd55b91",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13ac7f49",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e259ac3d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "319a33b8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e09c711",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "753919d2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37e7c827",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58840a11",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29e4ef56",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
